package experiments

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.SparkContext._

/**
  * Created by Alex on 2016/12/16.
  */
object WordCountDemo {
  def main(args:Array[String]): Unit ={
    val conf = new SparkConf().setAppName("WordCount").setMaster("local[*]")
    val sc = new SparkContext(conf)
    // change to hdfs urls
    val path = "E:\\workspaces\\eclipse\\Weka\\src\\weka\\associations\\*.java"
    val res = sc.textFile(path).flatMap(line =>{
      line.split("\\s+")
    }).map(x=>(x,1))
      .reduceByKey(_+_)
      .sortBy(x=>x._2,false)
      .collect()
    println("Total words " + res.length)
    res.foreach(x=>println("Word:%50s,Time:%5d".format(x._1,x._2)))
  }
}





